package com.chance.cc.crawler.development.scripts.xcar.sell;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import com.chance.cc.crawler.development.scripts.xcar.discounts.AkScript;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.script.*;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class AkIdCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AkScript.class);
    private static final String DOMAIN = "xcar";
    private static final String SIDE = "gather";
    private static final String RECORD_AGAIN_REQUEST = "record_again_request";//判断重新下载
    private static final String TURN_PAGE_ITEM_REQUEST = "turn_page_item_request";//判断重新下载
    private static final String KEYS = "keys";

    private static final String AKURL = "https://newcar.xcar.com.cn";

    private static final String AK = "https://newcar.xcar.com.cn/";
    private static final String AKSeries = "https://newcar.xcar.com.cn/car/0-0-\\S*";//品牌url
    private static final String AKModel = "https://newcar.xcar.com.cn/\\S*/";//车型url
    private static final String OLDYEAR = "https://newcar.xcar.com.cn/auto/index.php\\?r=newcar/SeriseParentIndex/\\S*";

    public Map<String, CompiledScript> compiledScriptMap;
    public ScriptEngine scriptEngine;
    @Override
    public String domain() {
        return DOMAIN;
    }
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return crawlerRecord.tagsCreator().bizTags().site().equals(SIDE);
    }
    @Override
    public void initUrlRegulars() {
        addUrlRegular(AK);
        addUrlRegular(AKSeries);
        addUrlRegular(AKModel);
        addUrlRegular(OLDYEAR);

    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords){
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        if (supportSourceRecords == null){
            return prepareLinks;
        }
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords){
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")){
                this.initKeyWord(crawlerRequestRecord,supportSourceRecord);
            }
        }
        return prepareLinks;
    }


    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type+".msg").get();
        if (!"success".equals(msg)){
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> size = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        // TODO 需要数据库接口来调取拿用
        for (String data : all){
            JSONObject jsonObject = JSONObject.parseObject(data);
            JSONObject keyword = jsonObject.getJSONObject("keyword");
            String brand = (String) keyword.get("brand");
            String brand_child = (String) keyword.get("brand_child");
            String series = (String) keyword.get("series");
            size.add(brand+"/"+brand_child+"/"+series);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(KEYS, size);
    }




    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parseLinks = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if ((!page.isDownloadSuccess()) || page.getStatusCode() != 200) {
            log.error("download page !=200 or page ==null", url, page.getStatusCode());
            if (page.getStatusCode() == 521) {
                initCompileScript();//初始化加密方式
                this.cookieUpdate(page,crawlerRecord,parseLinks);
            }
            this.recordAgainRequest(crawlerRecord, parseLinks);
            crawlerRecord.setNeedWashPage(false);//下载异常，不需要清洗页面
            return parseLinks;

        }
        if (url.matches(AK)){
            this.findBrandName(crawlerRecord,page,parseLinks);//查找品牌
        }
        if (url.matches(AKSeries)){
            this.findSeriesName(crawlerRecord,page,parseLinks);//查找车系
        }
        if(url.matches(AKModel) &&!url.matches(AKSeries)){//查找停售款
            this.sellOut(crawlerRecord,page,parseLinks);
        }

        return parseLinks;
    }

    /*
     * 查找品牌
     * */
    private void findBrandName(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        Map<String,String> map = new HashMap<>();
        List<String> nodes = page.getHtml().xpath("/html/body/script[15]").all();
        String brand =  null;
        String url  = null;
        for (String node : nodes){
            String s = node.replaceAll("&lt;", "")
                    .replaceAll("&quot;","")
                    .replaceAll("&gt;","")
                    .replaceAll(" ","").replaceAll("ahref=","").replaceAll("data-id=","");
            if (s.matches("/car/\\S*")){//url
                String[] split = s.split("/");
                url = AKURL +"/"+ split[1]+"/"+split[2]+"/";
            }else if(s.matches("/span\\S*")){//车牌
                brand = s.substring(5,s.length());
            }
            map.put(url,brand);//存入品牌和车系 一一对应
        }
        Set<String> strings = map.keySet();
        Iterator<String> iterator = strings.iterator();
        Map<String,String> brandMap = new HashMap<>();
        while(iterator.hasNext()){
            String key = iterator.next();
            String value = map.get(key);
            brandMap.put(value,key);
        }
        brandMap.remove("");

        //String startBrandName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("brand");
        Set<String> strings1 = brandMap.keySet();//map反转
        String brandUrl = null;
        String finalBrandUrl = null;
        KVTag startBrandAll = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(KEYS);
        List<String> startBrandList = (List<String>) startBrandAll.getVal();
        String fina = null;
        for (String s : startBrandList){
            String[] split = s.split("/");
            String startBrandName = split[0];
            if (fina.equals(startBrandName)){
                continue;
            }
            if (fina == null || !fina.equals(startBrandName) ){
                fina = startBrandName;
            }
            for (String key : brandMap.keySet()){
                if (key.equals(startBrandName)){
                    brandUrl = brandMap.get(key);//得到品牌链接
                    crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandName",startBrandName);//最终的品牌名
                    crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandUrl",brandUrl);//最终的品牌链接
                 }
            }
            if (StringUtils.isBlank(brandUrl)){
                log.error("series url is null"+brandUrl);
                return;
            }
            if (!finalBrandUrl.equals(brandUrl)){}
                CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRecord)
                        .httpUrl(brandUrl)
                        .recordKey(brandUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyResultTags()
                        .copyBizTags()
                        .build();
                parseLinks.add(record);
                continue;
            }
        }




    //查找车系and车型
    private void findSeriesName(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        /*String startSeriesName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("seriesId");
        String startCarModelName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("carModelId");*/
        String endBrandName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endBrandName");
        KVTag startBrandAll = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(KEYS);

        List<String> startBrandList = (List<String>) startBrandAll.getVal();
        String startSeriesName = null;
        String startCarModelName = null;
        for (String s : startBrandList) {
            String[] split = s.split("/");
            String startBrandName = split[0];
            if (endBrandName.equals(startBrandName)){
                startSeriesName = split[1];
                startCarModelName = split[2];
                List<Selectable> nodes = page.getHtml().xpath("/html/body/div[7]/div[2]/div[@class=\"choose_wrap mt10 clearfix\"]").nodes();
                String seriesName;//车系名
                String seriesUrl;//车系url
                String carModelName;//车型名称
                String carModelUrl = null;//车型链接
                for (Selectable series : nodes){
                    seriesName = series.xpath("./div[2]/a/text()").get();//车系名称
                    if (seriesName.equals(startSeriesName)){
                        seriesUrl = series.xpath("./div[2]/a/@href").get();
                        seriesUrl = AKURL + seriesUrl;//车系链接
                        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesName",startSeriesName);
                        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesUrl",seriesUrl);
                    }
                }
                for (Selectable carModel : nodes){
                    List<Selectable> nodesSon = carModel.xpath("./div[3]/div/div").nodes();
                    for (Selectable carModelSon : nodesSon){
                        carModelName = carModelSon.xpath("./div/div[1]/a/text()").get();//车型名称
                        carModelName = carModelName.replaceAll("・","·");
                        if (carModelName.equals(startCarModelName)){
                            carModelUrl = carModelSon.xpath("./div/div[1]/a/@href").get();
                            carModelUrl = AKURL + carModelUrl;//车型链接
                            crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelName",carModelName);
                            crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelUrl",carModelUrl);
                            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                    .itemPageRequest(crawlerRecord)
                                    .httpUrl(carModelUrl)
                                    .recordKey(carModelUrl)
                                    .copyBizTags()
                                    .copyResultTags()
                                    .releaseTime(System.currentTimeMillis())
                                    .build();
                            parseLinks.add(record);
                        }
                    }
                }
            }
        }


    }

    /*
    * 查找停售款
    * */
    private void sellOut(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        this.praiseForum(crawlerRecord,page);//清洗口碑和论坛链接
        String carModelUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelUrl");
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"serise_model_price_list\"]/div[1]/div[2]/div/ul/li").nodes();
        String sellout = null;
        for (Selectable node : nodes){
             sellout = node.xpath("./a/text()").get();
             sellout = sellout.substring(0,sellout.length()-1);//得到停售年份
            String[] split = carModelUrl.split("/");
            //得到停售款
            String ordSeriesUrl = "https://newcar.xcar.com.cn/auto/index.php?r=newcar/SeriseParentIndex/AjaxStopSaleModel&rand=0.5359516740196146&pserid="+split[3]+"&year="+sellout;
            CrawlerRequestRecord record =CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(ordSeriesUrl)
                    .recordKey(ordSeriesUrl)
                    .copyBizTags()
                    .copyResultTags()
                    .releaseTime(System.currentTimeMillis())
                    .build();
            Map<String,String> map = new HashMap<>();
            map.put("authority","newcar.xcar.com.cn");
            map.put("pragma","no-cache");
            map.put("cache-control","no-cache");
            map.put("sec-ch-ua","\"Chromium\";v=\"92\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"92\"");
            map.put("accept","*/*");
            map.put("x-requested-with","XMLHttpRequest");
            map.put("sec-ch-ua-mobile","?0");
            map.put("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36");
            map.put("sec-fetch-site","same-origin");
            map.put("sec-fetch-mode","cors");
            map.put("sec-fetch-dest","empty");
            map.put("referer","https://newcar.xcar.com.cn/"+split[3]+"/");
            map.put("accept-language","zh-CN,zh;q=0.9");
            map.put("cookie","_Xdwuv=610b87a9064c2; _Xdwnewuv=1; nguv=c_1628145577018713578218317218322554603; _PVXuv=610b87a87b0af; __jsluid_s=689fcf8e680f5a28101238dac0228d46; tel_cookie_uuuuuxcar=38214-34461-38664-34460; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1628145577,1628470765; fw_slc=1%3A1628470765%3B1%3A1628470768%3B1%3A1628470773%3B1%3A1628470776%3B1%3A1628470787; fw_clc=1%3A1628470768%3B1%3A1628470793%3B1%3A1628470930%3B1%3A1628470935%3B1%3A1628470941; fw_exc=1%3A1628470768%3B1%3A1628470935%3B1%3A1628470936%3B1%3A1628470941%3B1%3A1628470942; fw_pvc=1%3A1628470764%3B1%3A1628470768%3B1%3A1628470937%3B1%3A1628470942%3B1%3A1628470948; place_prid_lin=2; place_crid_lin=507; place_prname=%E4%B8%8A%E6%B5%B7%E5%B8%82; iwt_uuid=06c5744a-b2c6-4955-9d2c-1b4e08bf3ebc; __jsluid_h=548be05d72a992953f473c07ba84ddb9; place_crname=%E4%B8%8A%E6%B5%B7%E8%BD%A6%E5%B8%82; bbs_abtest=b; bbs_abtest_yj2=b; bbs_abtest_yj3=b; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; _newLocationInfo=%7B%22url%22%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2C%22city_id%22%3A%22507%22%2C%22province_id%22%3A%222%22%2C%20%22city_name%22%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; isRemoveCookie=1; zg_did=%7B%22did%22%3A%20%2217b1509ea2f782-0be36816a175a1-2343360-144000-17b1509ea30885%22%7D; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1628643931; place_prid=2; place_crid=507; place_ip=_1; zg_8f3d0255011c4bc5bae66beca6584825=%7B%22sid%22%3A%201628647157610%2C%22updated%22%3A%201628647257756%2C%22info%22%3A%201628145576504%2C%22superProperty%22%3A%20%22%7B%5C%22platform_type%5C%22%3A%20%5C%22PC%5C%22%2C%5C%22login_id%5C%22%3A%20null%2C%5C%22project_name%5C%22%3A%20%5C%22XCAR%5C%22%2C%5C%22login_status%5C%22%3A%200%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22firstScreen%22%3A%201628647157610%2C%22landHref%22%3A%20%22https%3A%2F%2Fnewcar.xcar.com.cn%2F8%2F%22%7D");
            record.getHttpRequest().setHeaders(map);
            parseLinks.add(record);
        }


    }

    private void praiseForum(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String endCarModelName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelName");
        String praiseName = null;//口碑
        String praiseUrl = null;
        String forumName = null;//论坛
        String forumUrl = null;
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"NavRoll\"]/ul/li").nodes();
        for (Selectable node:nodes){
            praiseName = node.xpath("./a/text()").get();
            if ("口碑".equals(praiseName)){
                praiseUrl = node.xpath("./a/@href").get();
                praiseUrl = AKURL + praiseUrl;
            }
            if ("论坛".equals(praiseName)){
                forumName = endCarModelName+praiseName;
                forumUrl = node.xpath("./a/@href").get();
                forumUrl = AKURL + forumUrl;
            }
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV("praiseUrl",praiseUrl);//口碑url
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumName",forumName);//论坛name
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumUrl",forumUrl);//论坛name
    }





    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerData = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();

            if (crawlerRecord.tagsCreator().resultTags().hasDataType(CrawlerEnum.CrawlerDataType.article)){
                if (url.matches(AKModel) && !url.matches(AKSeries)){
                this.washPageAll(crawlerRecord,page,crawlerData);
                }
                if (url.matches(OLDYEAR)){
                this.washPageSellOut(crawlerRecord,page,crawlerData);
                }
        }

        return crawlerData;
    }
    //清洗停售款
    private void washPageSellOut(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> crawlerData) {
        String endBrandName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endBrandName");//品牌name
        String endBrandUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endBrandUrl");//品牌url
        String endSeriesName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endSeriesName");//车系name
        String endSeriesUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endSeriesUrl");//车系url
        String endCarModelName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelName");//车型name
        String endCarModelUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelUrl");//车型url
        String praiseUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("praiseUrl");//口碑url
        String forumName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("forumName");//论坛nam
        String forumUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("forumUrl");//论坛url
        String url = crawlerRecord.getHttpRequest().getUrl();


        crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("praiseUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumUrl",null);

        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"ps_stop_model_list_table\"]/tbody/tr").nodes();
        String sellOutName = null;//停售
        String sellOutUrl = null;//停售
        crawlerRecord.tagsCreator().bizTags().addCustomKV(KEYS,null);
        for (Selectable node : nodes){
            String text = node.xpath("./td/p//text()").get();
            if (text != null){//判断不是第一行
                sellOutName = node.xpath("./td[1]/p/a//text()").get();//在售车型
                sellOutUrl = node.xpath("./td[1]/p/a/@href").get();//在售url
                sellOutUrl = AKURL + sellOutUrl;
                CrawlerData crawlerData1 = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(org.apache.commons.lang3.StringUtils.joinWith("-",crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article,System.currentTimeMillis() - (new Random().nextInt(10000))))
                        .url(url)
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .addContentKV(AutoVMCommonField.Field_Praise_url,praiseUrl)
                        .addContentKV(AutoVMCommonField.Field_Forum_name,forumName)
                        .addContentKV(AutoVMCommonField.Field_Forum_url,forumUrl)
                        .addContentKV(AutoVMCommonField.Field_Brand_name,endBrandName)
                        .addContentKV(AutoVMCommonField.Field_Brand_url,endBrandUrl)
                        .addContentKV(AutoVMCommonField.Tag_Field_Brand_Child,endSeriesName)
                        .addContentKV("brand_child_url",endSeriesUrl)
                        .addContentKV(AutoVMCommonField.Field_Series_name,endCarModelName)
                        .addContentKV(AutoVMCommonField.Field_Series_url,endCarModelUrl)
                        .addContentKV(AutoVMCommonField.Field_Model_name,sellOutName)
                        .addContentKV(AutoVMCommonField.Field_Model_url,sellOutUrl)
                        .addContentKV(AutoVMCommonField.Tag_Is_sale,"0")
                        .build();
                crawlerData.add(crawlerData1);

            }
        }

    }
    //获得在售款
    private void washPageAll(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> crawlerData) {
        String endBrandName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endBrandName");
        String endBrandUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endBrandUrl");
        String endSeriesName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endSeriesName");
        String endSeriesUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endSeriesUrl");
        String endCarModelName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelName");
        String endCarModelUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("endCarModelUrl");
        String praiseUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("praiseUrl");
        String forumName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("forumName");
        String forumUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("forumUrl");
        String url = crawlerRecord.getHttpRequest().getUrl();


        crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endBrandUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endSeriesUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("endCarModelUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("praiseUrl",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumName",null);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("forumUrl",null);
        String sell = page.getHtml().xpath("//*[@id=\"serise_model_price_list\"]/div[1]/ul/li/a/text()").get();//获得在售
        List<Selectable> sellNodes = page.getHtml().xpath("//*[@id=\"serise_model_price_list\"]/table/tbody/tr").nodes();
        crawlerRecord.tagsCreator().bizTags().addCustomKV(KEYS,null);
        if (sellNodes.size()==0){
            CrawlerData crawlerData1 = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(org.apache.commons.lang3.StringUtils.joinWith("-",crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article,System.currentTimeMillis() - (new Random().nextInt(10000))))
                    .url(url)
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV(AutoVMCommonField.Field_Praise_url,praiseUrl)
                    .addContentKV(AutoVMCommonField.Field_Forum_name,forumName)
                    .addContentKV(AutoVMCommonField.Field_Forum_url,forumUrl)
                    .addContentKV(AutoVMCommonField.Field_Brand_name,endBrandName)
                    .addContentKV(AutoVMCommonField.Field_Brand_url,endBrandUrl)
                    .addContentKV(AutoVMCommonField.Tag_Field_Brand_Child,endSeriesName)
                    .addContentKV("brand_child_url",endSeriesUrl)
                    .addContentKV(AutoVMCommonField.Field_Series_name,endCarModelName)
                    .addContentKV(AutoVMCommonField.Field_Series_url,endCarModelUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale,"0")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData1.setFilterPipelineResult(true);
            crawlerData.add(crawlerData1);
        }
        String sellName = null;
        String sellUrl = null;
        for (Selectable sellNode : sellNodes){
            String text = sellNode.xpath("./td/p//text()").get();
            if (text !=null ){//判断不是第一行
                sellName = sellNode.xpath("./td[1]/p/a//text()").get();//在售车型
                sellUrl = sellNode.xpath("./td[1]/p/a/@href").get();//在售url
                sellUrl = AKURL + sellUrl;
                CrawlerData crawlerData1 = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(org.apache.commons.lang3.StringUtils.joinWith("-",crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article,System.currentTimeMillis() - (new Random().nextInt(10000))))
                        .url(url)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .releaseTime(System.currentTimeMillis())
                        .addContentKV(AutoVMCommonField.Field_Praise_url,praiseUrl)
                        .addContentKV(AutoVMCommonField.Field_Forum_name,forumName)
                        .addContentKV(AutoVMCommonField.Field_Forum_url,forumUrl)
                        .addContentKV(AutoVMCommonField.Field_Brand_name,endBrandName)
                        .addContentKV(AutoVMCommonField.Field_Brand_url,endBrandUrl)
                        .addContentKV(AutoVMCommonField.Tag_Field_Brand_Child,endSeriesName)
                        .addContentKV("brand_child_url",endSeriesUrl)
                        .addContentKV(AutoVMCommonField.Field_Series_name,endCarModelName)
                        .addContentKV(AutoVMCommonField.Field_Series_url,endCarModelUrl)
                        .addContentKV(AutoVMCommonField.Field_Model_name,sellUrl)
                        .addContentKV(AutoVMCommonField.Field_Model_url,sellName)
                        .addContentKV(AutoVMCommonField.Tag_Is_sale,"1")
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData1.setFilterPipelineResult(true);
                crawlerData.add(crawlerData1);
            }
        }
    }


    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }
    /*
     * 下在失败 重新下载
     * */
    private void recordAgainRequest(CrawlerRequestRecord crawlerRecord, List<CrawlerRequestRecord> linksRecords) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(RECORD_AGAIN_REQUEST)){//判断是否重新下载过
            count = (int)crawlerBusinessTags.getCategoryTag().getKVTag(RECORD_AGAIN_REQUEST).getVal();
            if (count >= 5){
                log.error(DOMAIN + " download page the number of retries exceeds the limit,request url {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }
        String url = crawlerRecord.getHttpRequest().getUrl();
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get(TURN_PAGE_ITEM_REQUEST);
        CrawlerRequestRecord crawlerRequestRecord = null;
        if (type == null){
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }else{
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .copyBizTags()
                    .copyRequestTags()
                    .build();
        }
        if (crawlerRequestRecord == null){
            return;
        }
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(RECORD_AGAIN_REQUEST,count);
        linksRecords.add(crawlerRequestRecord);
    }


    private void cookieUpdate(HttpPage page, CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> parseLinks){
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpPage httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
        String js = httpPage.getRawText().substring(httpPage.getRawText().indexOf("("), httpPage.getRawText().lastIndexOf(")")+1);
        Map<String,String> cookieMap = new HashMap<>();
        try {
            String eval = (String) scriptEngine.eval(js);
            eval =  eval.substring(eval.indexOf("=")+1, eval.indexOf(";"));
            cookieMap.put("__jsl_clearance_s",eval);
            log.info("xcar cookie second update result __jsl_clearance_s={}",eval);
            crawlerRequestRecord.getHttpRequest().addHeader("cookie",processCookie(cookieMap));

            // 2. 生成 第二个 加密 cookie
            httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
            Matcher mtAuthor = Pattern.compile("};go\\((.*?)\\)</script>").matcher(httpPage.getRawText());
            if (mtAuthor.find()){
                String cookieGen = mtAuthor.group(1);
                Json cookieGenJson = new Json(cookieGen);
                String ha = cookieGenJson.jsonPath($_type + ".ha").get();
                log.info("xcar cookie cipher code {}",ha);
                compiledScriptMap.get(ha).eval();
                Invocable invocable = (Invocable) compiledScriptMap.get(ha).getEngine();
                String result = (String) invocable.invokeFunction("go", JSON.parseObject(cookieGen, Map.class));
                result =  result.substring(result.indexOf("=")+1, result.indexOf(";"));
                cookieMap.put("__jsl_clearance_s",result);
                result = processCookie(cookieMap);
                updateLocalAuthInfo(result);
                log.info("xcar cookie second update result {}",result);
            }

        } catch (Exception e) {
            log.error("xcar cookie update error {}",e.getMessage());
        }

    }
    private String processCookie(Map<String,String> cookieMap){
        StringBuffer cookieSB = new StringBuffer();
        for (Map.Entry<String, String> entry : cookieMap.entrySet()) {
            cookieSB.append(entry.getKey())
                    .append("=")
                    .append(entry.getValue())
                    .append(";");
        }
        return cookieSB.substring(0,cookieSB.lastIndexOf(";"));
    }
    private void initCompileScript(){
        if (compiledScriptMap == null){
            compiledScriptMap = new HashMap<>();
        }

        ScriptEngineManager sm = new ScriptEngineManager();
        NashornScriptEngineFactory factory = null;
        for (ScriptEngineFactory f : sm.getEngineFactories()) {
            if (f.getEngineName().equalsIgnoreCase("Oracle Nashorn")) {
                factory = (NashornScriptEngineFactory)f;
                break;
            }
        }
        String[] stringArray = ArrayUtils.toArray("-doe", "--global-per-engine");
        scriptEngine = factory.getScriptEngine(stringArray);
        List<String> ciphers = Arrays.asList("md5", "sha1", "sha256");
        //E:\chance-crawler-development\crawler-dev-bootstrap\src\main\java\com\chance\cc\crawler\development\bootstrap\xcar\js\
        //String filePathPrefix = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\xcar\\js\\";
        String filePathPrefix = "/data/chance_crawler_runner/domain/xcar/cipher_js/";
        for (String cipher : ciphers) {
            String filePath = filePathPrefix+cipher+".js";
            log.info("xcar cipher js file path {}",filePath);
            final CompiledScript compiled;
            try {
                compiled = ((Compilable)scriptEngine).compile(new FileReader(filePath));
                compiledScriptMap.put(cipher,compiled);
            } catch (ScriptException e) {
                e.printStackTrace();
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

        }
    }
}
